{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mlv7IfhFQhsv"
   },
   "source": [
    "# Data Perparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5He42w-lJcMZ",
    "outputId": "0b7b7284-39e0-45aa-c2bf-97bcb1502bc8"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'wget' is not recognized as an internal or external command,\n",
      "operable program or batch file.\n"
     ]
    }
   ],
   "source": [
    "!wget \"https://drive.google.com/u/0/uc?id=1rHb0FQ5z5ZpaY2HpyFGY6CeyDG0kTLoO&export=download\" -O english_python_data.txt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "T-mOljzdcIZL"
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'Field' from 'torchtext.data' (C:\\Users\\suche\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torchtext\\data\\__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 6\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01moptim\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01moptim\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Field, BucketIterator, Iterator\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchtext\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m data\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n",
      "\u001b[1;31mImportError\u001b[0m: cannot import name 'Field' from 'torchtext.data' (C:\\Users\\suche\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torchtext\\data\\__init__.py)"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "import torchtext\n",
    "from torchtext.data import Field, BucketIterator, Iterator\n",
    "from torchtext import data\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import random\n",
    "import math\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install wget\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "5WZWgUq7N0Qn",
    "outputId": "73a8933f-1c19-4a99-afc9-030c02235d35"
   },
   "outputs": [],
   "source": [
    "torchtext.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GIwv64O6zIeU",
    "outputId": "5b347bd0-f37c-4510-da59-f6d61ffa84cc"
   },
   "outputs": [],
   "source": [
    "%set_env CUDA_LAUNCH_BLOCKING = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c1UTpnAaSUlp"
   },
   "source": [
    "## Reading the text file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wYRncG2sPvbS"
   },
   "outputs": [],
   "source": [
    "f = open(\"english_python_data.txt\", \"r\",encoding=\"utf8\")\n",
    "file_lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wcPoWVXRYqbT",
    "outputId": "b8cde55f-6337-4c25-f148-1b4ce07b6f68"
   },
   "outputs": [],
   "source": [
    "file_lines[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wbhPC2wnSbOA"
   },
   "source": [
    "Our dataset is formulated in a manner where every question starts with '#'. Lines between two consecutive '#' forms the solution to the question."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "B1HR4YOwXXln"
   },
   "outputs": [],
   "source": [
    "dps = []\n",
    "dp = None\n",
    "for line in file_lines:\n",
    "  if line[0] == \"#\":\n",
    "    if dp:\n",
    "      dp['solution'] = ''.join(dp['solution'])\n",
    "      dps.append(dp)\n",
    "    dp = {\"question\": None, \"solution\": []}\n",
    "    dp['question'] = line[1:]\n",
    "  else:\n",
    "    dp[\"solution\"].append(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0VNitByXS7PF"
   },
   "source": [
    "Lets take a look at the first 50 entries of the data we have parsed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JOvv0T1FaH-V",
    "outputId": "590b87b5-2a87-4e19-e38e-ca6ae73a023c"
   },
   "outputs": [],
   "source": [
    "i=0\n",
    "for dp in dps:\n",
    "  print(\"\\n Question no: \", i+1)\n",
    "  i+=1\n",
    "  print(dp['question'][1:])\n",
    "  print(dp['solution'])\n",
    "  if i>49:\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qA-1MuOecZRt",
    "outputId": "fcf5ae3e-5417-4b77-917a-f05859a56659"
   },
   "outputs": [],
   "source": [
    "print(\"Dataset size:\", len(dps))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "swBfwcDaXEh5"
   },
   "source": [
    "## Using a custom tokenizer to tokenize python code\n",
    "\n",
    "Python is a programming language with its own unique syntax. Regular tokenizers like spacy are meant to tokenize english scentences and are not optimized towards Python's syntax. Here, we write our own custom tokenizer that makes use of Python's default [tokenize](https://docs.python.org/3/library/tokenize.html) library. When we make use of this library we only extract the token type and the token string.  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_UsevXhPgh1R"
   },
   "outputs": [],
   "source": [
    "from tokenize import tokenize, untokenize\n",
    "import io\n",
    "\n",
    "\n",
    "def tokenize_python_code(python_code_str):\n",
    "    python_tokens = list(tokenize(io.BytesIO(python_code_str.encode('utf-8')).readline))\n",
    "    tokenized_output = []\n",
    "    for i in range(0, len(python_tokens)):\n",
    "        tokenized_output.append((python_tokens[i].type, python_tokens[i].string))\n",
    "    return tokenized_output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2V-sePckv1-K",
    "outputId": "79ca4b9a-6222-4c0d-94fe-eb3a65b384df"
   },
   "outputs": [],
   "source": [
    "tokenized_sample = tokenize_python_code(dps[1]['solution'])\n",
    "print(tokenized_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dQ00a5cwwHtL",
    "outputId": "0f444d4d-ab03-472f-97dd-526e9fb46831"
   },
   "outputs": [],
   "source": [
    "print(untokenize(tokenized_sample).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4nsoc0eCpb8e"
   },
   "source": [
    "Since we have mere 5000 data points, we make use of data augmentations to increase the size of our dataset. While tokenizing the python code, we mask the names of certain variables randomly(with 'var_1, 'var_2' etc) to ensure that the model that we train does not merly fixate on the way the variables are named and actually tries to understand the inhrent logic and syntax of the python code.\n",
    "\n",
    "But, while randomly picking varibles to mask we avoid keyword literals(*keyword.kwlist*), control structures(as can be seen in below *skip_list*) and object properties. We add all such literals that need to be skipped into the *skip_list*\n",
    "\n",
    "```skip_list = ['range', 'enumerate', 'print', 'ord', 'int', 'float', 'char', 'list', 'dict', 'tuple', 'set', 'len', 'sum', 'min', 'max']```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lM-tqDwhYoVJ",
    "outputId": "f0bf41e6-0880-4ced-86c8-7eb6eb670792"
   },
   "outputs": [],
   "source": [
    "import keyword\n",
    "\n",
    "print(keyword.kwlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rbIMyrYapfwS"
   },
   "outputs": [],
   "source": [
    "def augment_tokenize_python_code(python_code_str, mask_factor=0.3):\n",
    "\n",
    "\n",
    "    var_dict = {} # Dictionary that stores masked variables\n",
    "\n",
    "    # certain reserved words that should not be treated as normal variables and\n",
    "    # hence need to be skipped from our variable mask augmentations\n",
    "    skip_list = ['range', 'enumerate', 'print', 'ord', 'int', 'float', 'zip'\n",
    "                 'char', 'list', 'dict', 'tuple', 'set', 'len', 'sum', 'min', 'max']\n",
    "    skip_list.extend(keyword.kwlist)\n",
    "\n",
    "    var_counter = 1\n",
    "    python_tokens = list(tokenize(io.BytesIO(python_code_str.encode('utf-8')).readline))\n",
    "    tokenized_output = []\n",
    "\n",
    "    for i in range(0, len(python_tokens)):\n",
    "      if python_tokens[i].type == 1 and python_tokens[i].string not in skip_list:\n",
    "        \n",
    "        if i>0 and python_tokens[i-1].string in ['def', '.', 'import', 'raise', 'except', 'class']: # avoid masking modules, functions and error literals\n",
    "          skip_list.append(python_tokens[i].string)\n",
    "          tokenized_output.append((python_tokens[i].type, python_tokens[i].string))\n",
    "        elif python_tokens[i].string in var_dict:  # if variable is already masked\n",
    "          tokenized_output.append((python_tokens[i].type, var_dict[python_tokens[i].string]))\n",
    "        elif random.uniform(0, 1) > 1-mask_factor: # randomly mask variables\n",
    "          var_dict[python_tokens[i].string] = 'var_' + str(var_counter)\n",
    "          var_counter+=1\n",
    "          tokenized_output.append((python_tokens[i].type, var_dict[python_tokens[i].string]))\n",
    "        else:\n",
    "          skip_list.append(python_tokens[i].string)\n",
    "          tokenized_output.append((python_tokens[i].type, python_tokens[i].string))\n",
    "      \n",
    "      else:\n",
    "        tokenized_output.append((python_tokens[i].type, python_tokens[i].string))\n",
    "    \n",
    "    return tokenized_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3HJwaBJOhdxb",
    "outputId": "3145786e-cf3e-4f6c-a230-6a714cde505f"
   },
   "outputs": [],
   "source": [
    "tokenized_sample = augment_tokenize_python_code(dps[1]['solution'])\n",
    "print(tokenized_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ciLm6oezs3A2",
    "outputId": "d8b1a093-c181-449f-8fa5-d9c59ae51498"
   },
   "outputs": [],
   "source": [
    "print(untokenize(tokenized_sample).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9hYArh3OznQ0"
   },
   "source": [
    "As one can see our augmented tokenizer picked num2 randomly and masked(replaced) it with by var_1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "shdWBNLu0DkK"
   },
   "source": [
    "## Building Train and Validation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mhBx_KFVaJIK"
   },
   "outputs": [],
   "source": [
    "python_problems_df = pd.DataFrame(dps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "id": "6i3TKNUoaWPC",
    "outputId": "fa0ee8e9-6893-4c67-8b63-456e15bc8fd5"
   },
   "outputs": [],
   "source": [
    "python_problems_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "A_93zNqfepxY",
    "outputId": "b9dd7668-095a-4a99-a29f-c0c861b302ac"
   },
   "outputs": [],
   "source": [
    "python_problems_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5cdHbscZsSJu"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(0)\n",
    "msk = np.random.rand(len(python_problems_df)) < 0.85 # Splitting data into 85% train and 15% validation\n",
    "\n",
    "train_df = python_problems_df[msk]\n",
    "val_df = python_problems_df[~msk]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jDtZSkLUu53D",
    "outputId": "a332b531-f0f2-4da7-b09c-0d09f3fd24cf"
   },
   "outputs": [],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c3SkbK0Qu9TZ",
    "outputId": "e5108fb5-fdb8-489f-ef92-d7f59ee5799c"
   },
   "outputs": [],
   "source": [
    "val_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6u7EyiRZ6brX"
   },
   "source": [
    "## Creating vocabulary using torchtext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b3ZQQWsWi4rn"
   },
   "source": [
    "In this section we will use torchtext Fields to construct the vocabulary for our sequence-to-sequence learning problem.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PlIi4zsLKwLE"
   },
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dLID_2ZtixKY"
   },
   "outputs": [],
   "source": [
    "Input = data.Field(tokenize = 'spacy',\n",
    "            init_token='<sos>', \n",
    "            eos_token='<eos>', \n",
    "            lower=True)\n",
    "\n",
    "Output = data.Field(tokenize = augment_tokenize_python_code,\n",
    "                    init_token='<sos>', \n",
    "                    eos_token='<eos>', \n",
    "                    lower=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ttEpbc2rixPx"
   },
   "outputs": [],
   "source": [
    "fields = [('Input', Input),('Output', Output)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mxsyNTBtogcD"
   },
   "source": [
    "Since our data augmentations have the potential to increase the vocabulary beyond what it initially is, we must ensure that we capture as many variations as possible in the vocabulary that we develop. In the the below code we apply our data augmentations 100 times to ensure that we can capture a majority of augmentations into our vocabulary. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uP_yJeqjoLLV"
   },
   "outputs": [],
   "source": [
    "train_example = []\n",
    "val_example = []\n",
    "\n",
    "train_expansion_factor = 100\n",
    "for j in range(train_expansion_factor):\n",
    "  for i in range(train_df.shape[0]):\n",
    "      try:\n",
    "          ex = data.Example.fromlist([train_df.question[i], train_df.solution[i]], fields)\n",
    "          train_example.append(ex)\n",
    "      except:\n",
    "          pass\n",
    "\n",
    "for i in range(val_df.shape[0]):\n",
    "    try:\n",
    "        ex = data.Example.fromlist([val_df.question[i], val_df.solution[i]], fields)\n",
    "        val_example.append(ex)\n",
    "    except:\n",
    "        pass       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ooT2EqpmtcAW"
   },
   "outputs": [],
   "source": [
    "train_data = data.Dataset(train_example, fields)\n",
    "valid_data =  data.Dataset(val_example, fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Uk4zu6Ntqgfr"
   },
   "outputs": [],
   "source": [
    "Input.build_vocab(train_data, min_freq = 0)\n",
    "Output.build_vocab(train_data, min_freq = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oA17CYnDJa4D",
    "outputId": "a6fccc99-87b2-46ee-d83a-c6a7b2e4bfa8"
   },
   "outputs": [],
   "source": [
    "Output.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h5f1HbzYuEcD"
   },
   "outputs": [],
   "source": [
    "def save_vocab(vocab, path):\n",
    "    import pickle\n",
    "    output = open(path, 'wb')\n",
    "    pickle.dump(vocab, output)\n",
    "    output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lN6tUrOxuFac"
   },
   "outputs": [],
   "source": [
    "# save_vocab(Input.vocab, \"/content/drive/MyDrive/TheSchoolOfAI/EndCapstone/src_vocab.pkl\")\n",
    "# save_vocab(Output.vocab, \"/content/drive/MyDrive/TheSchoolOfAI/EndCapstone/trg_vocab.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4pv-5RpJuAjb",
    "outputId": "315e1aec-e063-44a2-b91f-9d5437c0b1fd"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EgNa1xUluGHc",
    "outputId": "6de98ba1-7a01-4b4c-b071-05aeff2a97cd"
   },
   "outputs": [],
   "source": [
    "train_data[0].Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2sZedgyzu7cD",
    "outputId": "9fcc8c0d-1ff6-4867-d5fd-3213f62a5226"
   },
   "outputs": [],
   "source": [
    "print(vars(train_data.examples[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "njS_msug6eQ3"
   },
   "source": [
    "# Transformer Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bd-C0ntd8x1e"
   },
   "source": [
    "Our transformer can be understood in terms of its three components:\n",
    "1. An Encoder that encodes an input sequence into state representation vectors.\n",
    "2. An Attention mechanism that enables our Transformer model to focus on the right aspects of the sequential input stream. This is used repeatedly within both the encoder and the decoder to help them contextualize the input data.\n",
    "3. A Decoder that decodes the state representation vector to generate the target output sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cSjhBKP59ntL"
   },
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HgvD_MpkC2OS"
   },
   "source": [
    "![](https://raw.githubusercontent.com/bentrevett/pytorch-seq2seq/9479fcb532214ad26fd4bda9fcf081a05e1aaf4e/assets/transformer-encoder.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rQYN9WDA9Hh_"
   },
   "source": [
    "Our Encoder accepts a batch of source sequences and sequence masks as input. The source mask contains 1 in locations where the input sequence has valid values and 0 where the input sequence has <pad> values. This ensures that the attention mechanism within the encoder does not pay attention to <pad> values.\n",
    "\n",
    "We convert our source sequence tokens into embeddings(‘tok_embedding’) of ‘hid_dim’ length. Since were are not using any recurrent networks we need to tag each token with its positional indices in order to preserve sequential information. We create an indices tensor(i.e. ‘pos’) and convert this into an embedding(‘pos_embedding’) of length ‘hid_dim’. This is combined with the source sequence embeddings to create our initial Encoder Layer input tensor src. This src tensor is passed through a series of Encoder Layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NE6JimgOCz-w"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, \n",
    "                 input_dim, \n",
    "                 hid_dim, \n",
    "                 n_layers, \n",
    "                 n_heads, \n",
    "                 pf_dim,\n",
    "                 dropout, \n",
    "                 device,\n",
    "                 max_length = 1000):\n",
    "        super().__init__()\n",
    "\n",
    "        self.device = device\n",
    "        \n",
    "        self.tok_embedding = nn.Embedding(input_dim, hid_dim)\n",
    "        self.pos_embedding = nn.Embedding(max_length, hid_dim)\n",
    "        \n",
    "        self.layers = nn.ModuleList([EncoderLayer(hid_dim, \n",
    "                                                  n_heads, \n",
    "                                                  pf_dim,\n",
    "                                                  dropout, \n",
    "                                                  device) \n",
    "                                     for _ in range(n_layers)])\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\n",
    "        \n",
    "    def forward(self, src, src_mask):\n",
    "        \n",
    "        #src = [batch size, src len]\n",
    "        #src_mask = [batch size, 1, 1, src len]\n",
    "        \n",
    "        batch_size = src.shape[0]\n",
    "        src_len = src.shape[1]\n",
    "\n",
    "        pos = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)\n",
    "        \n",
    "        #pos = [batch size, src len]\n",
    "        src = self.dropout((self.tok_embedding(src) * self.scale) + self.pos_embedding(pos))\n",
    "        \n",
    "        #src = [batch size, src len, hid dim]\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            src = layer(src, src_mask)\n",
    "            \n",
    "        #src = [batch size, src len, hid dim]\n",
    "            \n",
    "        return src"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z1E12pQr9REU"
   },
   "source": [
    "An EncoderLayer is the basic building block of our Transformer’s Encoder component. Our src tensor along with its ‘src_mask’ are sent into a multi-head self-attention operation to help our model focus on the necessary aspects of the src tensor. The output from the attention operation is combined with the src tensor(via skip connection) and normalized to avoid vanishing/exploding gradients(during training). This combined output is sent into a PositionwiseFeedForwardLayer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2LheiXWVFDEg"
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self, \n",
    "                 hid_dim, \n",
    "                 n_heads, \n",
    "                 pf_dim,  \n",
    "                 dropout, \n",
    "                 device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
    "        self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
    "        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
    "        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, \n",
    "                                                                     pf_dim, \n",
    "                                                                     dropout)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src, src_mask):\n",
    "        \n",
    "        #src = [batch size, src len, hid dim]\n",
    "        #src_mask = [batch size, 1, 1, src len] \n",
    "                \n",
    "        #self attention\n",
    "        _src, _ = self.self_attention(src, src, src, src_mask)\n",
    "        \n",
    "        #dropout, residual connection and layer norm\n",
    "        src = self.self_attn_layer_norm(src + self.dropout(_src))\n",
    "        \n",
    "        #src = [batch size, src len, hid dim]\n",
    "        \n",
    "        #positionwise feedforward\n",
    "        _src = self.positionwise_feedforward(src)\n",
    "        \n",
    "        #dropout, residual and layer norm\n",
    "        src = self.ff_layer_norm(src + self.dropout(_src))\n",
    "        \n",
    "        #src = [batch size, src len, hid dim]\n",
    "        \n",
    "        return src"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6h_Iqnk4Jg5k"
   },
   "source": [
    "![](https://raw.githubusercontent.com/bentrevett/pytorch-seq2seq/9479fcb532214ad26fd4bda9fcf081a05e1aaf4e/assets/transformer-attention.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bjEVmTzG9uZ4"
   },
   "source": [
    "A PositionwiseFeedForwardLayer takes the combined input and processes it further using two fully connected layers and a Relu activation function between them. This in combination with the src embedding is the final output of an EncoderLayer. This process repeats for each EncoderLayer block.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R9w9xDUKL7LU"
   },
   "outputs": [],
   "source": [
    "class PositionwiseFeedforwardLayer(nn.Module):\n",
    "    def __init__(self, hid_dim, pf_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.fc_1 = nn.Linear(hid_dim, pf_dim)\n",
    "        self.fc_2 = nn.Linear(pf_dim, hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        #x = [batch size, seq len, hid dim]\n",
    "        \n",
    "        x = self.dropout(torch.relu(self.fc_1(x)))\n",
    "        \n",
    "        #x = [batch size, seq len, pf dim]\n",
    "        \n",
    "        x = self.fc_2(x)\n",
    "        \n",
    "        #x = [batch size, seq len, hid dim]\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VroGxzNo-GGI"
   },
   "source": [
    "## Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b3MtR2yd-Iaz"
   },
   "source": [
    "Attention is a mechanism that allows a model to focus on the necessary parts of the input sequence as per the demands of the task at hand.\n",
    "\n",
    "Researchers at google like to look at everything as an information retrieval problem. Therefore the [“Attention is all you need”](https://arxiv.org/abs/1706.03762) paper tries to look at attention in terms of “Query”, “Keys” and “Values”. A search engine accepts a “Query” and tries to match it up with Indices(i.e. Keys) in order to get appropriate values as results for the query. Similarly one can think of attention as a mechanism in which the query vector and key vector work towards getting the right attention weights(i.e. values).\n",
    "\n",
    "When multiple channels(or heads) of attention are applied in parallel to a single source, it is known as multi-head attention. This increases the learning capacity of the model and therefore leads to better results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZZmeHfGhGzkN"
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttentionLayer(nn.Module):\n",
    "    def __init__(self, hid_dim, n_heads, dropout, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        assert hid_dim % n_heads == 0\n",
    "        \n",
    "        self.hid_dim = hid_dim\n",
    "        self.n_heads = n_heads\n",
    "        self.head_dim = hid_dim // n_heads\n",
    "        \n",
    "        self.fc_q = nn.Linear(hid_dim, hid_dim)\n",
    "        self.fc_k = nn.Linear(hid_dim, hid_dim)\n",
    "        self.fc_v = nn.Linear(hid_dim, hid_dim)\n",
    "        \n",
    "        self.fc_o = nn.Linear(hid_dim, hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)\n",
    "        \n",
    "    def forward(self, query, key, value, mask = None):\n",
    "        \n",
    "        batch_size = query.shape[0]\n",
    "        \n",
    "        #query = [batch size, query len, hid dim]\n",
    "        #key = [batch size, key len, hid dim]\n",
    "        #value = [batch size, value len, hid dim]\n",
    "                \n",
    "        Q = self.fc_q(query)\n",
    "        K = self.fc_k(key)\n",
    "        V = self.fc_v(value)\n",
    "        \n",
    "        #Q = [batch size, query len, hid dim]\n",
    "        #K = [batch size, key len, hid dim]\n",
    "        #V = [batch size, value len, hid dim]\n",
    "                \n",
    "        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        \n",
    "        #Q = [batch size, n heads, query len, head dim]\n",
    "        #K = [batch size, n heads, key len, head dim]\n",
    "        #V = [batch size, n heads, value len, head dim]\n",
    "                \n",
    "        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale\n",
    "        \n",
    "        #energy = [batch size, n heads, query len, key len]\n",
    "        \n",
    "        if mask is not None:\n",
    "            energy = energy.masked_fill(mask == 0, -1e10)\n",
    "        \n",
    "        attention = torch.softmax(energy, dim = -1)\n",
    "                \n",
    "        #attention = [batch size, n heads, query len, key len]\n",
    "                \n",
    "        x = torch.matmul(self.dropout(attention), V)\n",
    "        \n",
    "        #x = [batch size, n heads, query len, head dim]\n",
    "        \n",
    "        x = x.permute(0, 2, 1, 3).contiguous()\n",
    "        \n",
    "        #x = [batch size, query len, n heads, head dim]\n",
    "        \n",
    "        x = x.view(batch_size, -1, self.hid_dim)\n",
    "        \n",
    "        #x = [batch size, query len, hid dim]\n",
    "        \n",
    "        x = self.fc_o(x)\n",
    "        \n",
    "        #x = [batch size, query len, hid dim]\n",
    "        \n",
    "        return x, attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZBFqGg5z-o_r"
   },
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YbTr7YPSMRpC"
   },
   "source": [
    "![](https://raw.githubusercontent.com/bentrevett/pytorch-seq2seq/9479fcb532214ad26fd4bda9fcf081a05e1aaf4e/assets/transformer-decoder.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SaHaOGNm_Isc"
   },
   "source": [
    "The architecture of a Decoder is very similar to that of the encoder with the significant differences resulting from the presence of input from two sources, the target sequence and the state representation vector from the encoder. Much like how we had an EncoderLayer block for Encoder, we will be having a DecoderLayer that accepts as input the combination of the embedding from the target token sequence(tok_embedding) and embedding of positional indices for these tokens. And as mentioned earlier, the encoder’s output also acts as one of the inputs to the DecoderLayer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iWBMMF45MMNS"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, \n",
    "                 output_dim, \n",
    "                 hid_dim, \n",
    "                 n_layers, \n",
    "                 n_heads, \n",
    "                 pf_dim, \n",
    "                 dropout, \n",
    "                 device,\n",
    "                 max_length = 10000):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.device = device\n",
    "        \n",
    "        self.tok_embedding = nn.Embedding(output_dim, hid_dim)\n",
    "        self.pos_embedding = nn.Embedding(max_length, hid_dim)\n",
    "        \n",
    "        self.layers = nn.ModuleList([DecoderLayer(hid_dim, \n",
    "                                                  n_heads, \n",
    "                                                  pf_dim, \n",
    "                                                  dropout, \n",
    "                                                  device)\n",
    "                                     for _ in range(n_layers)])\n",
    "        \n",
    "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\n",
    "        \n",
    "    def forward(self, trg, enc_src, trg_mask, src_mask):\n",
    "        \n",
    "        #trg = [batch size, trg len]\n",
    "        #enc_src = [batch size, src len, hid dim]\n",
    "        #trg_mask = [batch size, 1, trg len, trg len]\n",
    "        #src_mask = [batch size, 1, 1, src len]\n",
    "                \n",
    "        batch_size = trg.shape[0]\n",
    "        trg_len = trg.shape[1]\n",
    "        \n",
    "        pos = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)\n",
    "                            \n",
    "        #pos = [batch size, trg len]\n",
    "\n",
    "        trg = self.dropout((self.tok_embedding(trg) * self.scale) + self.pos_embedding(pos))\n",
    "                \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            trg, attention = layer(trg, enc_src, trg_mask, src_mask)\n",
    "        \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "        #attention = [batch size, n heads, trg len, src len]\n",
    "        \n",
    "        output = self.fc_out(trg)\n",
    "        \n",
    "        #output = [batch size, trg len, output dim]\n",
    "            \n",
    "        return output, attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bcK1-4qZ_WBM"
   },
   "source": [
    "The DecoderLayer forms the building block of our Transformer’s decoder. Each DecoderLayer involves two attention operations:\n",
    "1. Self-attention on trg embedding.\n",
    "2. Multi-head attention operation that uses the trg as query vector and the encoder outputs act as the key and value vectors.\n",
    "\n",
    "The presence of an extra Multi-head attention operation differentiates the DecoderLayer from an EncoderLayer.\n",
    "\n",
    "The attention outputs from self-attention are normalized and combined with the trg embedding using a residual connection. This is then sent into the multi-head attention operation along with the encoder outputs. The attention layer outputs are then combined with the trg input again and normalized before sending it into the position-wise feedforward layer to generate the final outputs of the DecoderLayer.\n",
    "\n",
    "The purpose of all normalization operations is to prevent vanishing/exploding gradients during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CMEr1IFUMxco"
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self, \n",
    "                 hid_dim, \n",
    "                 n_heads, \n",
    "                 pf_dim, \n",
    "                 dropout, \n",
    "                 device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
    "        self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
    "        self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
    "        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
    "        self.encoder_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
    "        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, \n",
    "                                                                     pf_dim, \n",
    "                                                                     dropout)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, trg, enc_src, trg_mask, src_mask):\n",
    "        \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "        #enc_src = [batch size, src len, hid dim]\n",
    "        #trg_mask = [batch size, 1, trg len, trg len]\n",
    "        #src_mask = [batch size, 1, 1, src len]\n",
    "        \n",
    "        #self attention\n",
    "        _trg, _ = self.self_attention(trg, trg, trg, trg_mask)\n",
    "        \n",
    "        #dropout, residual connection and layer norm\n",
    "        trg = self.self_attn_layer_norm(trg + self.dropout(_trg))\n",
    "            \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "            \n",
    "        #encoder attention\n",
    "        _trg, attention = self.encoder_attention(trg, enc_src, enc_src, src_mask)\n",
    "        # query, key, value\n",
    "        \n",
    "        #dropout, residual connection and layer norm\n",
    "        trg = self.enc_attn_layer_norm(trg + self.dropout(_trg))\n",
    "                    \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "        \n",
    "        #positionwise feedforward\n",
    "        _trg = self.positionwise_feedforward(trg)\n",
    "        \n",
    "        #dropout, residual and layer norm\n",
    "        trg = self.ff_layer_norm(trg + self.dropout(_trg))\n",
    "        \n",
    "        #trg = [batch size, trg len, hid dim]\n",
    "        #attention = [batch size, n heads, trg len, src len]\n",
    "        \n",
    "        return trg, attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "udpPhQ2UN8oQ"
   },
   "source": [
    "The main class that implements a transformer for seq2seq problems is given below.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Dr3Mg8OGN6ul"
   },
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, \n",
    "                 encoder, \n",
    "                 decoder, \n",
    "                 src_pad_idx, \n",
    "                 trg_pad_idx, \n",
    "                 device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_pad_idx = src_pad_idx\n",
    "        self.trg_pad_idx = trg_pad_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def make_src_mask(self, src):\n",
    "        \n",
    "        #src = [batch size, src len]\n",
    "        \n",
    "        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)\n",
    "\n",
    "        #src_mask = [batch size, 1, 1, src len]\n",
    "\n",
    "        return src_mask\n",
    "    \n",
    "    def make_trg_mask(self, trg):\n",
    "        \n",
    "        #trg = [batch size, trg len]\n",
    "        \n",
    "        trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)\n",
    "        \n",
    "        #trg_pad_mask = [batch size, 1, 1, trg len]\n",
    "        \n",
    "        trg_len = trg.shape[1]\n",
    "        \n",
    "        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device = self.device)).bool()\n",
    "        \n",
    "        #trg_sub_mask = [trg len, trg len]\n",
    "            \n",
    "        trg_mask = trg_pad_mask & trg_sub_mask\n",
    "        \n",
    "        #trg_mask = [batch size, 1, trg len, trg len]\n",
    "        \n",
    "        return trg_mask\n",
    "\n",
    "    def forward(self, src, trg):\n",
    "        \n",
    "        #src = [batch size, src len]\n",
    "        #trg = [batch size, trg len]\n",
    "                \n",
    "        src_mask = self.make_src_mask(src)\n",
    "        trg_mask = self.make_trg_mask(trg)\n",
    "        \n",
    "        #src_mask = [batch size, 1, 1, src len]\n",
    "        #trg_mask = [batch size, 1, trg len, trg len]\n",
    "        \n",
    "        enc_src = self.encoder(src, src_mask)\n",
    "        \n",
    "        #enc_src = [batch size, src len, hid dim]\n",
    "                \n",
    "        output, attention = self.decoder(trg, enc_src, trg_mask, src_mask)\n",
    "        \n",
    "        #output = [batch size, trg len, output dim]\n",
    "        #attention = [batch size, n heads, trg len, src len]\n",
    "        \n",
    "        return output, attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SvrK6zbF_2Fs"
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4zsZjSSWOSHc"
   },
   "outputs": [],
   "source": [
    "INPUT_DIM = len(Input.vocab)\n",
    "OUTPUT_DIM = len(Output.vocab)\n",
    "HID_DIM = 256\n",
    "ENC_LAYERS = 3\n",
    "DEC_LAYERS = 3\n",
    "ENC_HEADS = 16\n",
    "DEC_HEADS = 16\n",
    "ENC_PF_DIM = 512\n",
    "DEC_PF_DIM = 512\n",
    "ENC_DROPOUT = 0.1\n",
    "DEC_DROPOUT = 0.1\n",
    "\n",
    "enc = Encoder(INPUT_DIM, \n",
    "              HID_DIM, \n",
    "              ENC_LAYERS, \n",
    "              ENC_HEADS, \n",
    "              ENC_PF_DIM, \n",
    "              ENC_DROPOUT, \n",
    "              device)\n",
    "\n",
    "dec = Decoder(OUTPUT_DIM, \n",
    "              HID_DIM, \n",
    "              DEC_LAYERS, \n",
    "              DEC_HEADS, \n",
    "              DEC_PF_DIM, \n",
    "              DEC_DROPOUT, \n",
    "              device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "x3vG-tI737e6",
    "outputId": "67f5f267-5b3e-40e3-80b8-da8b818c759a"
   },
   "outputs": [],
   "source": [
    "len(Output.vocab.__dict__['freqs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iYVZYDVcOUGK"
   },
   "outputs": [],
   "source": [
    "SRC_PAD_IDX = Input.vocab.stoi[Input.pad_token]\n",
    "TRG_PAD_IDX = Output.vocab.stoi[Output.pad_token]\n",
    "\n",
    "model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qd0ePzj0OzLa",
    "outputId": "75e4ffdb-e5fb-4c9b-ba35-5c59d62868ff"
   },
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fmZ0hyo8O0vE"
   },
   "outputs": [],
   "source": [
    "def initialize_weights(m):\n",
    "    if hasattr(m, 'weight') and m.weight.dim() > 1:\n",
    "        nn.init.xavier_uniform_(m.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NRtAM9Y4O2N2"
   },
   "outputs": [],
   "source": [
    "model.apply(initialize_weights);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NEpApG3YO3ZE"
   },
   "outputs": [],
   "source": [
    "LEARNING_RATE = 0.0005\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "95cvaGRg_-Ml"
   },
   "source": [
    "## Loss function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bs2XdrveCtDD"
   },
   "source": [
    "We have used augmentations in our dataset to mask variable literals. This means that our model can predict a variety of values for a particular variable and all of them are correct as long as the predictions are consistent through the code. This would mean that our training labels are not very certain and hence it would make more sense to treat them to be correct with probability ```1- smooth_eps``` and incorrect otherwise. This is what label smoothening enables us to do. The following is the implementation of CrossEntropyLoss with label smoothening."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NNfxmsLxD7yb"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import math\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class CrossEntropyLoss(nn.CrossEntropyLoss):\n",
    "    \"\"\"CrossEntropyLoss - with ability to recieve distrbution as targets, and optional label smoothing\"\"\"\n",
    "\n",
    "    def __init__(self, weight=None, ignore_index=-100, reduction='mean', smooth_eps=None, smooth_dist=None, from_logits=True):\n",
    "        super(CrossEntropyLoss, self).__init__(weight=weight,\n",
    "                                               ignore_index=ignore_index, reduction=reduction)\n",
    "        self.smooth_eps = smooth_eps\n",
    "        self.smooth_dist = smooth_dist\n",
    "        self.from_logits = from_logits\n",
    "\n",
    "    def forward(self, input, target, smooth_dist=None):\n",
    "        if smooth_dist is None:\n",
    "            smooth_dist = self.smooth_dist\n",
    "        return cross_entropy(input, target, weight=self.weight, ignore_index=self.ignore_index,\n",
    "                             reduction=self.reduction, smooth_eps=self.smooth_eps,\n",
    "                             smooth_dist=smooth_dist, from_logits=self.from_logits)\n",
    "\n",
    "\n",
    "def cross_entropy(inputs, target, weight=None, ignore_index=-100, reduction='mean',\n",
    "                  smooth_eps=None, smooth_dist=None, from_logits=True):\n",
    "    \"\"\"cross entropy loss, with support for target distributions and label smoothing https://arxiv.org/abs/1512.00567\"\"\"\n",
    "    smooth_eps = smooth_eps or 0\n",
    "\n",
    "    # ordinary log-liklihood - use cross_entropy from nn\n",
    "    if _is_long(target) and smooth_eps == 0:\n",
    "        if from_logits:\n",
    "            return F.cross_entropy(inputs, target, weight, ignore_index=ignore_index, reduction=reduction)\n",
    "        else:\n",
    "            return F.nll_loss(inputs, target, weight, ignore_index=ignore_index, reduction=reduction)\n",
    "\n",
    "    if from_logits:\n",
    "        # log-softmax of inputs\n",
    "        lsm = F.log_softmax(inputs, dim=-1)\n",
    "    else:\n",
    "        lsm = inputs\n",
    "\n",
    "    masked_indices = None\n",
    "    num_classes = inputs.size(-1)\n",
    "\n",
    "    if _is_long(target) and ignore_index >= 0:\n",
    "        masked_indices = target.eq(ignore_index)\n",
    "\n",
    "    if smooth_eps > 0 and smooth_dist is not None:\n",
    "        if _is_long(target):\n",
    "            target = onehot(target, num_classes).type_as(inputs)\n",
    "        if smooth_dist.dim() < target.dim():\n",
    "            smooth_dist = smooth_dist.unsqueeze(0)\n",
    "        target.lerp_(smooth_dist, smooth_eps)\n",
    "\n",
    "    if weight is not None:\n",
    "        lsm = lsm * weight.unsqueeze(0)\n",
    "\n",
    "    if _is_long(target):\n",
    "        eps_sum = smooth_eps / num_classes\n",
    "        eps_nll = 1. - eps_sum - smooth_eps\n",
    "        likelihood = lsm.gather(dim=-1, index=target.unsqueeze(-1)).squeeze(-1)\n",
    "        loss = -(eps_nll * likelihood + eps_sum * lsm.sum(-1))\n",
    "    else:\n",
    "        loss = -(target * lsm).sum(-1)\n",
    "\n",
    "    if masked_indices is not None:\n",
    "        loss.masked_fill_(masked_indices, 0)\n",
    "\n",
    "    if reduction == 'sum':\n",
    "        loss = loss.sum()\n",
    "    elif reduction == 'mean':\n",
    "        if masked_indices is None:\n",
    "            loss = loss.mean()\n",
    "        else:\n",
    "            loss = loss.sum() / float(loss.size(0) - masked_indices.sum())\n",
    "\n",
    "    return loss\n",
    "\n",
    "\n",
    "def onehot(indexes, N=None, ignore_index=None):\n",
    "    \"\"\"\n",
    "    Creates a one-representation of indexes with N possible entries\n",
    "    if N is not specified, it will suit the maximum index appearing.\n",
    "    indexes is a long-tensor of indexes\n",
    "    ignore_index will be zero in onehot representation\n",
    "    \"\"\"\n",
    "    if N is None:\n",
    "        N = indexes.max() + 1\n",
    "    sz = list(indexes.size())\n",
    "    output = indexes.new().byte().resize_(*sz, N).zero_()\n",
    "    output.scatter_(-1, indexes.unsqueeze(-1), 1)\n",
    "    if ignore_index is not None and ignore_index >= 0:\n",
    "        output.masked_fill_(indexes.eq(ignore_index).unsqueeze(-1), 0)\n",
    "    return output\n",
    "\n",
    "def _is_long(x):\n",
    "    if hasattr(x, 'data'):\n",
    "        x = x.data\n",
    "    return isinstance(x, torch.LongTensor) or isinstance(x, torch.cuda.LongTensor)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RnWtafKlAhb_"
   },
   "outputs": [],
   "source": [
    "def maskNLLLoss(inp, target, mask):\n",
    "    # print(inp.shape, target.shape, mask.sum())\n",
    "    nTotal = mask.sum()\n",
    "    crossEntropy = CrossEntropyLoss(ignore_index = TRG_PAD_IDX, smooth_eps=0.20)\n",
    "    loss = crossEntropy(inp, target)\n",
    "    loss = loss.to(device)\n",
    "    return loss, nTotal.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "n9Dy_wWrO46l"
   },
   "outputs": [],
   "source": [
    "criterion = maskNLLLoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BQGdcEFmAxlO"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "itmoGDi_tN74"
   },
   "source": [
    "In order to re-apply our augmentations differently in every epoch we re-create our dataset and dataloaders at the start of each epoch. This regularizes our training process and helps us come up with better models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ycBBiEpuO6cG"
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def make_trg_mask(trg):\n",
    "        \n",
    "        #trg = [batch size, trg len]\n",
    "        \n",
    "        trg_pad_mask = (trg != TRG_PAD_IDX).unsqueeze(1).unsqueeze(2)\n",
    "        \n",
    "        #trg_pad_mask = [batch size, 1, 1, trg len]\n",
    "        \n",
    "        trg_len = trg.shape[1]\n",
    "        \n",
    "        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device = device)).bool()\n",
    "        \n",
    "        #trg_sub_mask = [trg len, trg len]\n",
    "            \n",
    "        trg_mask = trg_pad_mask & trg_sub_mask\n",
    "        \n",
    "        #trg_mask = [batch size, 1, trg len, trg len]\n",
    "        \n",
    "        return trg_mask\n",
    "\n",
    "def train(model, iterator, optimizer, criterion, clip):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    n_totals = 0\n",
    "    print_losses = []\n",
    "    for i, batch in tqdm(enumerate(iterator), total=len(iterator)):\n",
    "        # print(batch)\n",
    "        loss = 0\n",
    "        src = batch.Input.permute(1, 0)\n",
    "        trg = batch.Output.permute(1, 0)\n",
    "        trg_mask = make_trg_mask(trg)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        output, _ = model(src, trg[:,:-1])\n",
    "                \n",
    "        #output = [batch size, trg len - 1, output dim]\n",
    "        #trg = [batch size, trg len]\n",
    "            \n",
    "        output_dim = output.shape[-1]\n",
    "            \n",
    "        output = output.contiguous().view(-1, output_dim)\n",
    "        trg = trg[:,1:].contiguous().view(-1)\n",
    "                \n",
    "        #output = [batch size * trg len - 1, output dim]\n",
    "        #trg = [batch size * trg len - 1]\n",
    "            \n",
    "        mask_loss, nTotal = criterion(output, trg, trg_mask)\n",
    "        \n",
    "        mask_loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        print_losses.append(mask_loss.item() * nTotal)\n",
    "        n_totals += nTotal\n",
    "\n",
    "\n",
    "        \n",
    "    return sum(print_losses) / n_totals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zi3Ev8gaO79_"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    n_totals = 0\n",
    "    print_losses = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in tqdm(enumerate(iterator), total=len(iterator)):\n",
    "\n",
    "            src = batch.Input.permute(1, 0)\n",
    "            trg = batch.Output.permute(1, 0)\n",
    "            trg_mask = make_trg_mask(trg)\n",
    "\n",
    "            output, _ = model(src, trg[:,:-1])\n",
    "            \n",
    "            #output = [batch size, trg len - 1, output dim]\n",
    "            #trg = [batch size, trg len]\n",
    "            \n",
    "            output_dim = output.shape[-1]\n",
    "            \n",
    "            output = output.contiguous().view(-1, output_dim)\n",
    "            trg = trg[:,1:].contiguous().view(-1)\n",
    "            \n",
    "            #output = [batch size * trg len - 1, output dim]\n",
    "            #trg = [batch size * trg len - 1]\n",
    "            \n",
    "            mask_loss, nTotal = criterion(output, trg, trg_mask)\n",
    "\n",
    "            print_losses.append(mask_loss.item() * nTotal)\n",
    "            n_totals += nTotal\n",
    "\n",
    "        \n",
    "    return sum(print_losses) / n_totals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JuB4JqQRO9Wg"
   },
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aax76Ie4O_Cr",
    "outputId": "fbc1062f-f4d9-4ff4-887f-a365835c3de6"
   },
   "outputs": [],
   "source": [
    "N_EPOCHS = 50\n",
    "CLIP = 1\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_example = []\n",
    "    val_example = []\n",
    "\n",
    "    for i in range(train_df.shape[0]):\n",
    "        try:\n",
    "            ex = data.Example.fromlist([train_df.question[i], train_df.solution[i]], fields)\n",
    "            train_example.append(ex)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    for i in range(val_df.shape[0]):\n",
    "        try:\n",
    "            ex = data.Example.fromlist([val_df.question[i], val_df.solution[i]], fields)\n",
    "            val_example.append(ex)\n",
    "        except:\n",
    "            pass       \n",
    "\n",
    "    train_data = data.Dataset(train_example, fields)\n",
    "    valid_data =  data.Dataset(val_example, fields)\n",
    "\n",
    "    BATCH_SIZE = 16\n",
    "    train_iterator, valid_iterator = BucketIterator.splits((train_data, valid_data), batch_size = BATCH_SIZE, \n",
    "                                                                sort_key = lambda x: len(x.Input),\n",
    "                                                                sort_within_batch=True, device = device)\n",
    "\n",
    "    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)\n",
    "    valid_loss = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    \n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), '/content/drive/MyDrive/TheSchoolOfAI/EndCapstone/model.pt')\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MMKkB13yPeJ7"
   },
   "outputs": [],
   "source": [
    "SRC = Input\n",
    "TRG = Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f3_aq7QTPBFc",
    "outputId": "e8bdc0e6-cd7e-4d5e-bb92-1029c59ef9d6"
   },
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('/content/drive/MyDrive/TheSchoolOfAI/EndCapstone/model.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ieIjql9uPKH1"
   },
   "outputs": [],
   "source": [
    "def translate_sentence(sentence, src_field, trg_field, model, device, max_len = 50000):\n",
    "    \n",
    "    model.eval()\n",
    "        \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en')\n",
    "        tokens = [token.text.lower() for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token.lower() for token in sentence]\n",
    "\n",
    "    tokens = [src_field.init_token] + tokens + [src_field.eos_token]\n",
    "        \n",
    "    src_indexes = [src_field.vocab.stoi[token] for token in tokens]\n",
    "\n",
    "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)\n",
    "    \n",
    "    src_mask = model.make_src_mask(src_tensor)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        enc_src = model.encoder(src_tensor, src_mask)\n",
    "\n",
    "    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]\n",
    "\n",
    "    for i in range(max_len):\n",
    "\n",
    "        trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
    "\n",
    "        trg_mask = model.make_trg_mask(trg_tensor)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)\n",
    "        \n",
    "        pred_token = output.argmax(2)[:,-1].item()\n",
    "        \n",
    "        trg_indexes.append(pred_token)\n",
    "\n",
    "        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:\n",
    "            break\n",
    "    \n",
    "    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]\n",
    "    \n",
    "    return trg_tokens[1:], attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1mlHzigPJpLT"
   },
   "source": [
    "## Displaying Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WTBipndfPOlX"
   },
   "outputs": [],
   "source": [
    "def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\n",
    "    \n",
    "    assert n_rows * n_cols == n_heads\n",
    "    \n",
    "    fig = plt.figure(figsize=(30,50))\n",
    "    \n",
    "    for i in range(n_heads):\n",
    "        \n",
    "        ax = fig.add_subplot(n_rows, n_cols, i+1)\n",
    "        \n",
    "        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\n",
    "\n",
    "        cax = ax.matshow(_attention, cmap='bone')\n",
    "\n",
    "        ax.tick_params(labelsize=12)\n",
    "        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \n",
    "                           rotation=45)\n",
    "        ax.set_yticklabels(['']+translation)\n",
    "\n",
    "        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sIyQeFx-dMlK",
    "outputId": "a8b2ce67-54a4-4298-f00b-4a6bcdfae999"
   },
   "outputs": [],
   "source": [
    "src = \"write a function that adds two numbers\"\n",
    "src=src.split(\" \")\n",
    "translation, attention = translate_sentence(src, SRC, TRG, model, device)\n",
    "\n",
    "print(f'predicted trg sequence: ')\n",
    "print(translation)\n",
    "print(\"code: \\n\", untokenize(translation[:-1]).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CHWqhmvtPTJv",
    "outputId": "71e4428d-392a-4532-ae56-257747c55279"
   },
   "outputs": [],
   "source": [
    "display_attention(src, translation, attention)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xhIC2mgfv37B"
   },
   "source": [
    "# Sample Outputs for English to Python translation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-lwaEkWxK8DA"
   },
   "source": [
    "Lets load our pretrained model to perform inference on a set of examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h_d2MUwHK62D",
    "outputId": "fec0c908-0a5a-4ea1-e318-e85762a5f927"
   },
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('/content/drive/MyDrive/TheSchoolOfAI/EndCapstone/model_ep_0_pt_2.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vveLENmdLEE4"
   },
   "source": [
    "Function that translates an English src string to python code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hiYFFxnzj6Mn"
   },
   "outputs": [],
   "source": [
    "def eng_to_python(src):\n",
    "  src=src.split(\" \")\n",
    "  translation, attention = translate_sentence(src, SRC, TRG, model, device)\n",
    "\n",
    "  print(f'predicted trg: \\n')\n",
    "  # print(translation)\n",
    "  print(untokenize(translation[:-1]).decode('utf-8'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TG3TsKgQWy4p"
   },
   "outputs": [],
   "source": [
    "SRC = Input\n",
    "TRG = Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EBOoET5H6km5"
   },
   "source": [
    "## Example 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tuNEA7wzijhY",
    "outputId": "0eee0f83-b6d0-46c1-921c-f7b486ff1623"
   },
   "outputs": [],
   "source": [
    "src = \"program to sort a list of dictionaries by key\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-FfPrg0l7ZNM"
   },
   "source": [
    "## Example 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lHOL5t7SjVeh",
    "outputId": "13322cb2-1500-4934-9727-c0df6232fefe"
   },
   "outputs": [],
   "source": [
    "src = \"function to merge two lists\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GzJqBaYp7deR"
   },
   "source": [
    "## Example 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8MuBCMCSjsR4",
    "outputId": "33c2dbac-3aff-48fb-e7a8-923bc0429273"
   },
   "outputs": [],
   "source": [
    "src = \"program to find gcd\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7SnTjVI87vAp"
   },
   "source": [
    "## Example 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vhq7YCN-kifr",
    "outputId": "50b27c35-7414-4c81-f751-e5309b5b870b"
   },
   "outputs": [],
   "source": [
    "src = \"program to calculate simple interest\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-USiTjDL7ydL"
   },
   "source": [
    "## Example 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8NO84A1ZlSJt",
    "outputId": "ae516f68-cce1-4db6-fae4-ef4f6e1c57a8"
   },
   "outputs": [],
   "source": [
    "src = \"function to sum odd elements of list\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fud3AeSZ71ft"
   },
   "source": [
    "## Example 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YOAWbmjfld7A",
    "outputId": "b51327e6-af22-441f-de4a-9bc56bc3a4fd"
   },
   "outputs": [],
   "source": [
    "src = \"program to multiply integers in a list\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Aj3PvhIZ8GFN"
   },
   "source": [
    "## Example 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mtwY2d_819Oi",
    "outputId": "0bfa3d0f-df3e-4e4d-d480-d33c37195356"
   },
   "outputs": [],
   "source": [
    "src = \"program to reverse a string\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tblyRJt58sO2"
   },
   "source": [
    "## Example 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GqEI8sqch-ms",
    "outputId": "0260f5f9-e072-4921-f4fd-b1a9b2da1646"
   },
   "outputs": [],
   "source": [
    "src = \"function to find reverse of a string\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bddDdZgk-AfF"
   },
   "source": [
    "## Example 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Krlfk6MxrHg8",
    "outputId": "0570785d-1054-4fad-fdd6-2e990d355018"
   },
   "outputs": [],
   "source": [
    "src = \"program to find the length of tuple\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XSv3wBwQ-IWJ"
   },
   "source": [
    "## Example 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KcoA-tgmrHlz",
    "outputId": "dfed0cfd-61e7-4d1e-b6e7-902a4e879769"
   },
   "outputs": [],
   "source": [
    "src = \"program to find the area of a square\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LkuQGH9T-NTn"
   },
   "source": [
    "## Example 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "z6ef4xQJrHrR",
    "outputId": "98678841-5c38-4061-94cd-dff426ace8f6"
   },
   "outputs": [],
   "source": [
    "src = \"program to print epoch timestamp\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6viTGKHA-Pte"
   },
   "source": [
    "## Example 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o1K0vG_Uh-6T",
    "outputId": "2a43447b-c87a-4e94-84b1-7eb1ac57b562"
   },
   "outputs": [],
   "source": [
    "src = \"program to find ascii value of charecter\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j1uI-bk0_xXj"
   },
   "source": [
    "## Example 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "04a9ORHq52NO",
    "outputId": "369f7dfd-75cd-4895-e038-cb36d0a0892a"
   },
   "outputs": [],
   "source": [
    "src = \"function to find fibonacci sequence\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oqst2jY0A8da"
   },
   "source": [
    "## Example 14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XOmQTvVe6BdW",
    "outputId": "eedccb72-7996-4e2e-cf89-bb8a2efdd6a5"
   },
   "outputs": [],
   "source": [
    "src = \"function to find largest element in a dictionary\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LrzUWvbPA_yN"
   },
   "source": [
    "## Example 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GfikZjRnA6in",
    "outputId": "b1a07736-6a8c-40a9-bb32-18781ce0eefd"
   },
   "outputs": [],
   "source": [
    "src = \"program to convert list of tuples into a dictionary\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TwGLVjeJBEse"
   },
   "source": [
    "## Example 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EyjQlqrlBbd_",
    "outputId": "c607e278-935e-4a41-8e52-be636c29c591"
   },
   "outputs": [],
   "source": [
    "src = \"function to convert to binary\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bCp9MFXuBz5f"
   },
   "source": [
    "## Example 17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "P92rLpfzB34-",
    "outputId": "3868f44b-39de-4ccc-a27b-ed88bdd973c7"
   },
   "outputs": [],
   "source": [
    "src = \"program to implement a linked list\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vCzyMpcZB4M4"
   },
   "source": [
    "## Example 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JHvANCDoB7fu",
    "outputId": "fdb9bb36-0c31-401f-fde3-faeb2880e6b5"
   },
   "outputs": [],
   "source": [
    "src = \"program to add lists elementwise\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sTWtnH5oEAnG"
   },
   "source": [
    "## Example 19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "azuLJKtGEClE",
    "outputId": "b7bc923a-b9cf-435c-cb1f-189ad47892d0"
   },
   "outputs": [],
   "source": [
    "src = \"program to find common values between two sets\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Bwss8KejEDIX"
   },
   "source": [
    "## Example 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-4tnH_2qEDff",
    "outputId": "d1bbaeeb-8b91-4a6f-e8c5-79af7300903e"
   },
   "outputs": [],
   "source": [
    "src = \"program to find number of unique values in a list\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YUrPbMgHEDtK"
   },
   "source": [
    "## Example 21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dYFV_FFYED81",
    "outputId": "6fca285c-6ed9-44a7-c6ad-140e82e916dc"
   },
   "outputs": [],
   "source": [
    "src = \"function to remove empty lists from a list of lists\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cwD42Lm5Ek3p"
   },
   "source": [
    "## Example 22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LQHIADDYEnH6",
    "outputId": "1ba3d2ec-1245-4501-bcd9-1ea353219ce4"
   },
   "outputs": [],
   "source": [
    "src = \"write a function to capitalize a string\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ffn0hW_yEnuq"
   },
   "source": [
    "## Example 23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9Vdu1QDbEq7a",
    "outputId": "82eb2d04-c24c-4658-dbef-d91f0366a03f"
   },
   "outputs": [],
   "source": [
    "src = \"write a function to find the area of a circle\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-Xo7Je-HErTH"
   },
   "source": [
    "## Example 24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qn4-1CW_EuRT",
    "outputId": "dbfcb747-f71c-4d51-8583-853d72321bc5"
   },
   "outputs": [],
   "source": [
    "src = \"write a python program to merge two dictionaries\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4QbqAX4cEurC"
   },
   "source": [
    "## Example 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9dxvGAM0Ewvp",
    "outputId": "e6c06519-55ed-4f38-a15b-7c9369c4c9bb"
   },
   "outputs": [],
   "source": [
    "src = \"write a function to find factorial\"\n",
    "\n",
    "eng_to_python(src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MYIOx1D-KIZ9"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "English to Python.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  },
  "vscode": {
   "interpreter": {
    "hash": "7e30d2e47ca7336ca37b5e80ff437f78e4048eb30f7a1781c4481edfe7bd7fff"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
